﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
using System.Collections;

namespace TextMining
{
    public class BlogReader   // reader of the blog data set
    {
        // a set containing the stop words
        private HashSet<string> _stopWords = new HashSet<string>();

        private Regex _postReader = new Regex(@"[A-Za-z]+");
        public int Documents { private set; get; }  // number of dataset's documents

        private string ReadLine(StreamReader reader)   // read a blog post
        {
            string[] delims = { ":&:" };  // delimeter

            string line = reader.ReadLine();  // read line
            if (line == null)   // EOF found
                return null;
            // split all parts
            string[] sections = line.Split(delims, StringSplitOptions.RemoveEmptyEntries);

            return sections[4];   // the 4th part is the post
        }

        public void ReadStopWords(string filename)  // read the stop words
        {
            using (StreamReader reader = new StreamReader(filename))
            {
                string line;
                Regex regex = new Regex(@"\w+");
                Match match;

                while ((line = reader.ReadLine()) != null)   // for all lines
                {
                    match = regex.Match(line);   // parse line in order to get a word
                    if (match != null && match.Success == true)  // if a word was found
                        _stopWords.Add(match.Groups[0].Value);  // add it
                }
            }
        }

        public void ReadBlog(string filename, WordsSet set)   // Reads the blog and prepares words' values
        {
            string line;
            Word word;
            Document document;

            Documents = 0;
            using (StreamReader reader = new StreamReader(filename))
            {
                while ((line = ReadLine(reader)) != null)  // for all posts
                {
                    Documents++;  // a document is a blog's line

                    document = new Document(Documents);  // new document
                    Match match = _postReader.Match(line);   // parse post to find all words

                    while (match != null && match.Success == true)  // for all valid words found
                    {
                        if (_stopWords.Contains(match.Groups[0].Value))  // if it's not a new word
                        {
                            match = match.NextMatch();   // find the next one
                            continue;
                        }
                        word = set.Add(match.Groups[0].Value);  // else, add it
                        word.AddOccurence(document);   // one more occurence for this word

                        match = match.NextMatch();   // find the next word
                    }
                }
            }
        }
    }
}
